#!/usr/bin/env python
import spade, urllib2, time, sys, sqlite3, re, string, random
from bs4 import BeautifulSoup, Comment


class reviewCrawler(spade.Agent.Agent):
	count = 0

	class inform(spade.Behaviour.PeriodicBehaviour):
		def _onTick(self):
			print "------------Statistics-------------"
			print "Reviews processed: " + str(self.myAgent.count)
			print "-----------------------------------"

	class informWhitepages(spade.Behaviour.OneShotBehaviour):
		def _process(self):					# inform service directory
			sd = spade.DF.ServiceDescription()
			sd.setName("review")
			sd.setType("crawler")
			dad = spade.DF.DfAgentDescription()
			dad.addService(sd)
			dad.setAID(self.myAgent.getAID())
			res = self.myAgent.registerService(dad)
			print "Informed Directory Services"

	class askforURL(spade.Behaviour.PeriodicBehaviour):
		def _onTick(self):
			dad = spade.DF.DfAgentDescription()
			ds = spade.DF.ServiceDescription()
			ds.setType("provider")
			ds.setName("reviewurl")
			dad.addService(ds)
			agents = self.myAgent.searchService(dad)
			if len(agents) > 0:
				request = spade.ACLMessage.ACLMessage()
				request.setPerformative("request")
				request.setOntology("review")
				request.addReceiver(random.choice(agents).getAID())
				self.myAgent.send(request)
			else: print "no data agent active"

	class parseURL(spade.Behaviour.EventBehaviour):
		def _process(self):					# receive a request to crawl a review
			self.myAgent.count += 1
			Hotel_Rating = "0"
			Review_Rating = "0"
			Quote = "Empty"
			Text = "No data found"
			Star = "0"
			Hotel_Subratings = dict()
			Review_Subratings = dict()
			Hotel_Subratings['Cleanliness'] = "0"
			Hotel_Subratings['Service'] = "0"
			Hotel_Subratings['Location'] = "0"
			Hotel_Subratings['Rooms'] = "0"
			Hotel_Subratings['Sleep Quality'] = "0"
			Hotel_Subratings['Value'] = "0"
			Review_Subratings['Cleanliness'] = "0"
			Review_Subratings['Service'] = "0"
			Review_Subratings['Location'] = "0"
			Review_Subratings['Rooms'] = "0"
			Review_Subratings['Sleep Quality'] = "0"
			Review_Subratings['Value'] = "0"
			msg = self._receive(True)
			self.myAgent.coordinator = msg.getSender()
			basic_url = msg.getContent()
			print "Reading review: " + basic_url
			try:
				soop = BeautifulSoup(urllib2.urlopen("http://www.tripadvisor.com"+basic_url).read(),'html.parser')
				Hotel_id = basic_url.split('-')[2]
				Review_id = basic_url.split('-')[3]
				Review = soop.find('div',{'id':"review_" + Review_id[1:]})
			except: print "could not read page"
			try:
				Hotel_Rating = soop.find("div","userRating").img['alt'].split()[0].replace(',','.')
				Quote = Review.find("div",{"class":"quote"}).text
			except: print "rating, quote"
			try:
				Text = Review.find("div",{"class":"entry"}).p.text.strip()
				Star = soop.find("div","locationContent").img['alt'].split()[0].replace(',','.')
			except: print "text, star"
			try:
				for subrating in soop.find('div', {'id':'SUMMARYBOX'}).findAll('li'):
					Hotel_Subratings[subrating.text.strip()] = subrating.img['alt'].split()[0].replace(',','.')
			except: print "subratings"
			try:
				Review_Rating = Review.find("div",{"class":"rating"}).img['alt'].split()[0].replace(',','.')
				for subrating in Review.find("ul",{"class":"recommend"}).findAll("li"):
					Review_Subratings[subrating.text.strip()] = subrating.img['alt'].split()[0].replace(',','.')
			except: print "review and subratings"
			try:
				Data = [Review_id, Hotel_id, Star, Hotel_Rating,
					Hotel_Subratings['Cleanliness'], Hotel_Subratings['Service'], Hotel_Subratings['Location'],
					Hotel_Subratings['Rooms'], Hotel_Subratings['Sleep Quality'], Hotel_Subratings['Value'],
					Quote, Text, Review_Rating,
					Review_Subratings['Cleanliness'], Review_Subratings['Service'], Review_Subratings['Location'],
					Review_Subratings['Rooms'], Review_Subratings['Sleep Quality'], Review_Subratings['Value']]
				message = '&#%'.join(Data)
				print "Found review: " + Review_id
				#print Data
				reply = msg.createReply()
				reply.setProtocol(basic_url)
				reply.setOntology("review")
				reply.setPerformative("inform")
				reply.setContent(filter(lambda x: x in string.printable, message))
				self.myAgent.send(reply)
				print "Sent reply for: " + basic_url
			except:
				print "data assignment and message creation"
			print "---------------------------------------------------------------"

	def _setup(self):
		request_template = spade.Behaviour.ACLTemplate()
		request_template.setOntology("review")
		request_template.setPerformative("request")
		self.addBehaviour(self.parseURL(),spade.Behaviour.MessageTemplate(request_template))
		self.addBehaviour(self.informWhitepages(),None)
		#self.addBehaviour(self.inform(180),None)
		self.addBehaviour(self.inform(60),None)
		self.addBehaviour(self.askforURL(2),None)

if __name__ == "__main__":
	try: name = sys.argv[1]
	except: name = "reviewcrawrler"
	a = reviewCrawler(name+"@127.0.0.1","secret")
	print "Started Review Crawler with name: " + name
	a.start()
	Alive = True
	while Alive:
		try: time.sleep(1)
		except KeyboardInterrupt: Alive = False
	a.stop()
	sys.exit(0)



